#!/usr/bin/env python
# coding: utf-8

# In[1]:


#from PIL import Image,ImageDraw
#import PIL.Image as IMG

import urllib as urllib
import re

import sqlite3 as sqlite

from bs4 import BeautifulSoup as BeautifulSoup

#from urlparse import urljoin
from urllib.parse import urljoin  # as urljoin
#urllib.parse  

##from pysqlite2 import dbapi2 as sqlite

import nn
mynet=nn.searchnet('nn.db')

# Create a list of words to ignore
ignorewords={'the':1,'of':1,'to':1,'and':1,'a':1,'in':1,'is':1,'it':1}

class crawler:
	# Initialize the crawler with the name of database
	def __init__(self,dbname):
		self.con=sqlite.connect(dbname)
        #self.createindextables(self)
	
	def __del__(self):
		self.con.close()

	def dbcommit(self):
		self.con.commit()

	# Auxilliary function for getting an entry id and adding 
	# it if it's not present
	def getentryid(self,table,field,value,createnew=True):
		cur=self.con.execute(
		"select rowid from %s where %s='%s'" % (table,field,value))
		res=cur.fetchone()
		if res==None:
			cur=self.con.execute(
			"insert into %s (%s) values ('%s')" % (table,field,value))
			return cur.lastrowid
		else:
			return res[0] 


	# Index an individual page
	def addtoindex(self,url,soup):
		if self.isindexed(url): return
		print ('Indexing '+url)
	
		# Get the individual words
		text=self.gettextonly(soup)
		words=self.separatewords(text)
		
		# Get the URL id
		urlid=self.getentryid('urllist','url',url)
		
		# Link each word to this url
		for i in range(len(words)):
			word=words[i]
			if word in ignorewords: continue
			wordid=self.getentryid('wordlist','word',word)
			self.con.execute("insert into wordlocation(urlid,wordid,location) values (%d,%d,%d)" % (urlid,wordid,i))
	

	
	# Extract the text from an HTML page (no tags)
	def gettextonly(self,soup):
		v=soup.string
		if v==None:   ##original Null
			c=soup.contents
			resulttext=''
			for t in c:
				subtext=self.gettextonly(t)
				resulttext+=subtext+'\n'
			return resulttext
		else:
			return v.strip()

	# Seperate the words by any non-whitespace character
	def separatewords(self,text):
		splitter=re.compile('\\W*')
		return [s.lower() for s in splitter.split(text) if s!='']

		
	# Return true if this url is already indexed
	def isindexed(self,url):
		return False
	
	# Add a link between two pages
	def addlinkref(self,urlFrom,urlTo,linkText):
		words=self.separatewords(linkText)
		fromid=self.getentryid('urllist','url',urlFrom)
		toid=self.getentryid('urllist','url',urlTo)
		if fromid==toid: return
		cur=self.con.execute("insert into link(fromid,toid) values (%d,%d)" % (fromid,toid))
		linkid=cur.lastrowid
		for word in words:
			if word in ignorewords: continue
			wordid=self.getentryid('wordlist','word',word)
			self.con.execute("insert into linkwords(linkid,wordid) values (%d,%d)" % (linkid,wordid))

	# Starting with a list of pages, do a breadth
	# first search to the given depth, indexing pages
	# as we go
	def crawl(self,pages,depth=2):
		for i in range(depth):
			print('depth %d begins' % i)
			newpages={}
			for page in pages:
				try:
					c=urllib.request.urlopen(page) # .read()
					print(c)
				except:
					print ("Could not open %s" % page)
					continue
				try:
					soup=BeautifulSoup(c.read())
					self.addtoindex(page,soup)
	
					links=soup('a')
					for link in links:
						if ('href' in dict(link.attrs)):
							url=urljoin(page,link['href'])
							if url.find("'")!=-1: continue
							url=url.split('#')[0]  # remove location portion
							if url[0:4]=='http' and not self.isindexed(url):
								newpages[url]=1
							linkText=self.gettextonly(link)
							self.addlinkref(page,url,linkText)
	
					self.dbcommit()
				except:
					print ("Could not parse page %s" % page)

			pages=newpages

	
	# Create the database tables
	def createindextables(self): 
		self.con.execute('create table urllist(url)')
		self.con.execute('create table wordlist(word)')
		self.con.execute('create table wordlocation(urlid,wordid,location)')
		self.con.execute('create table link(fromid integer,toid integer)')
		self.con.execute('create table linkwords(wordid,linkid)')
		self.con.execute('create index wordidx on wordlist(word)')
		self.con.execute('create index urlidx on urllist(url)')
		self.con.execute('create index wordurlidx on wordlocation(wordid)')
		self.con.execute('create index urltoidx on link(toid)')
		self.con.execute('create index urlfromidx on link(fromid)')
		self.dbcommit()

	def calculatepagerank(self,iterations=20):
		# clear out the current page rank tables
		self.con.execute('drop table if exists pagerank')
		self.con.execute('create table pagerank(urlid primary key,score)')
		
		# initialize every url with a page rank of 1
		for (urlid,) in self.con.execute('select rowid from urllist'):
			self.con.execute('insert into pagerank(urlid,score) values (%d,1.0)' % urlid)
		self.dbcommit()
		
		for i in range(iterations):
			print ("Iteration %d" % (i))
			for (urlid,) in self.con.execute('select rowid from urllist'):
				pr=0.15
				
				# Loop through all the pages that link to this one
				for (linker,) in self.con.execute(
				'select distinct fromid from link where toid=%d' % urlid):
					# Get the page rank of the linker
					linkingpr=self.con.execute(
					'select score from pagerank where urlid=%d' % linker).fetchone()[0]

					# Get the total number of links from the linker
					linkingcount=self.con.execute(
					'select count(*) from link where fromid=%d' % linker).fetchone()[0]
					pr+=0.85*(linkingpr/linkingcount)
				self.con.execute(
				'update pagerank set score=%f where urlid=%d' % (pr,urlid))
			self.dbcommit()

class searcher:
	def __init__(self,dbname):
		self.con=sqlite.connect(dbname)

	def __del__(self):
		self.con.close()

	def getmatchrows(self,q):
		# Strings to build the query
		fieldlist='w0.urlid'
		tablelist=''  
		clauselist=''
		wordids=[]

		# Split the words by spaces
		words=q.split(' ')  
		tablenumber=0

		for word in words:
			# Get the word ID
			wordrow=self.con.execute(
			"select rowid from wordlist where word='%s'" % word).fetchone()
			if wordrow!=None:
				wordid=wordrow[0]
				wordids.append(wordid)
				if tablenumber>0:
					tablelist+=','
					clauselist+=' and '
					clauselist+='w%d.urlid=w%d.urlid and ' % (tablenumber-1,tablenumber)
				fieldlist+=',w%d.location' % tablenumber
				tablelist+='wordlocation w%d' % tablenumber      
				clauselist+='w%d.wordid=%d' % (tablenumber,wordid)
				tablenumber+=1

        # Create the query from the separate parts
		try:
			fullquery='select %s from %s where %s' % (fieldlist,tablelist,clauselist)
			print ('fullquery:',fullquery)

			cur=self.con.execute(fullquery)
			rows=[row for row in cur]

			return rows,wordids
		except:
			return [],wordids

	def getscoredlist(self,rows,wordids):
		totalscores=dict([(row[0],0) for row in rows])

		# This is where we'll put our scoring functions
		weights=[(1.0,self.locationscore(rows)), 
						 (1.0,self.frequencyscore(rows)),
						 (1.0,self.pagerankscore(rows)),
						 (1.0,self.linktextscore(rows,wordids)),
						 (5.0,self.nnscore(rows,wordids))]
		for (weight,scores) in weights:
			for url in totalscores:
				totalscores[url]+=weight*scores[url]

		return totalscores

	def geturlname(self,id):
		return self.con.execute(
		"select url from urllist where rowid=%d" % id).fetchone()[0]

	def query(self,q):
		rows,wordids=self.getmatchrows(q)
		scores=self.getscoredlist(rows,wordids)
		rankedscores=[(score,url) for (url,score) in scores.items()]
		rankedscores.sort()
		rankedscores.reverse()
		for (score,urlid) in rankedscores[0:10]:
			print( '%f\t%s' % (score,self.geturlname(urlid)))
		return wordids,[r[1] for r in rankedscores[0:10]]

	def normalizescores(self,scores,smallIsBetter=0):
		vsmall=0.00001 # Avoid division by zero errors
		if smallIsBetter:
			minscore=min(scores.values())
			return dict([(u,float(minscore)/max(vsmall,l)) for (u,l) in scores.items()])
		else:
			maxscore=max(scores.values())
			if maxscore==0: maxscore=vsmall
			return dict([(u,float(c)/maxscore) for (u,c) in scores.items()])

	def frequencyscore(self,rows):
		counts=dict([(row[0],0) for row in rows])
		for row in rows: counts[row[0]]+=1
		return self.normalizescores(counts)

	def locationscore(self,rows):
		locations=dict([(row[0],1000000) for row in rows])
		for row in rows:
			loc=sum(row[1:])
			if loc<locations[row[0]]: locations[row[0]]=loc
		
		return self.normalizescores(locations,smallIsBetter=1)

	def distancescore(self,rows):
		# If there's only one word, everyone wins!
		if len(rows[0])<=2: return dict([(row[0],1.0) for row in rows])

		# Initialize the dictionary with large values
		mindistance=dict([(row[0],1000000) for row in rows])

		for row in rows:
			dist=sum([abs(row[i]-row[i-1]) for i in range(2,len(row))])
			if dist<mindistance[row[0]]: mindistance[row[0]]=dist
		return self.normalizescores(mindistance,smallIsBetter=1)

	def inboundlinkscore(self,rows):
		uniqueurls=dict([(row[0],1) for row in rows])
		inboundcount=dict([(u,self.con.execute('select count(*) from link where toid=%d' % u).fetchone()[0]) for u in uniqueurls])   
		return self.normalizescores(inboundcount)

	def linktextscore(self,rows,wordids):
		linkscores=dict([(row[0],0) for row in rows])
		for wordid in wordids:
			cur=self.con.execute('select link.fromid,link.toid from linkwords,link where wordid=%d and linkwords.linkid=link.rowid' % wordid)
			for (fromid,toid) in cur:
				if toid in linkscores:
					pr=self.con.execute('select score from pagerank where urlid=%d' % fromid).fetchone()[0]
					linkscores[toid]+=pr
		maxscore=max(linkscores.values())
		normalizedscores=dict([(u,float(l)/maxscore) for (u,l) in linkscores.items()])
		return normalizedscores

	def pagerankscore(self,rows):
		pageranks=dict([(row[0],self.con.execute('select score from pagerank where urlid=%d' % row[0]).fetchone()[0]) for row in rows])
		maxrank=max(pageranks.values())
		normalizedscores=dict([(u,float(l)/maxrank) for (u,l) in pageranks.items()])
		return normalizedscores

	def nnscore(self,rows,wordids):
		# Get unique URL IDs as an ordered list
		urlids=[urlid for urlid in dict([(row[0],1) for row in rows])]
		nnres=mynet.getresult(wordids,urlids)
		scores=dict([(urlids[i],nnres[i]) for i in range(len(urlids))])
		return self.normalizescores(scores)

